# -*- coding: utf-8 -*-
'''
Created on Aug 5, 2012

@author: LONG HOANG GIANG
'''

import sys, os
from urlparse import urljoin
import traceback
sys.path.append(os.path.expanduser('/home5/vietcntt/longhoanggiang/python'))
sys.path.append(os.path.join(os.path.dirname(__file__), '../'))
from CrawlerLib import Http, commonlib, html2text
import datetime
import re
import time
from lxml import etree
import cPickle as pickle
import sqlite3 as db

INCLUDE_TITLE = False

def text2html(text):
    text = re.sub(r"\n{2,}", "\n", text.strip())
    text = re.sub(r"\n", "<br />", text.strip())
    return text

def getChapterLink():
    url = 'http://vozforums.com/showthread.php?t=2786623'
    print '> get chapters url: {0}'.format(url)
    tree = Http.getXMLTree(url)
    items = []
    extra = []
    for item in tree.xpath("//div[@id='post_message_46837917']//a"):
        name = commonlib.stringify(item)
        link = commonlib.normalize_str(item.get('href'))
        if link == '': continue
        link = urljoin(url, link)
        if ('Ngoại truyện' in name) or ('Special Chapter' in name): 
            extra.append({'title': name, 'url': link})
            continue
        items.append({'title': name, 'url': link})
    for item in extra:
        items.append(item)
    return items

def getContent(item):
    url = item['url']
    print '---- get content {0}: {1}'.format(item['title'], url)
    chapterId = commonlib.extractText(r"showpost.php\?p=(\d+)&postcount=", url, 1)
    chapters = []
    for itime in range(0,10):
        try:
            tree = Http.getXMLTree(url)
            if tree == None: raise Exception, 'Load Html and build tree failed'
            contentNode = tree.xpath("//table[@id='post{0}']/tr[3]//div[@id='post_message_{0}']".format(chapterId))
            content = html2text.html2text(etree.tounicode(contentNode[0]), 'http://vozforums.com/', True, True, False, True)
            content = text2html(content)
            arr = []
            for i in re.finditer(r"(<b>Chap.+?</b>)", content, re.IGNORECASE):
                arr.append(i.group(1))
            for i in re.finditer(r"(<b>Special Chapter.+?</b>)", content, re.IGNORECASE):
                arr.append(i.group(1))
            for i in re.finditer(r"(<b>Ngoại truyện chương.+?</b>)".decode('utf-8'), content, re.IGNORECASE):
                print i.group(1)
                print content
                arr.append(i.group(1))
                
            firstIdx = 0
            lastIdx = 0
            if len(arr) > 0:
                for i in range(0, len(arr)+1):
                    if i == 0: continue
                    if i < len(arr):
                        pos = content.find(arr[i])
                        lastIdx = pos
                    else:
                        lastIdx = len(content)
                    detail = content[firstIdx:lastIdx]
                    title = commonlib.extractText(r"<b>(Chap.+?)</b>", detail, 1)
                    if re.search(r"Ngoại truyện chương.+?".decode('utf-8'), detail):
                        title = commonlib.extractText(r"<b>(Ngoại truyện chương.+?)</b>".decode('utf-8'), detail, 1)
                        title = re.sub(r"Ngoại truyện chương".decode('utf-8'), "NGOẠI TRUYỆN CHƯƠNG", title)
                    elif re.search(r"Special Chapter.+?", detail):
                        title = commonlib.extractText(r"<b>(Special Chapter.+?)</b>", detail, 1)
                    title = re.sub(r"<.+?>", "", title)
                    title = re.sub(r"[:_#-]+", "", title)
                    if detail == '': continue
#                    print title
#                    print detail
                    chapters.append({'title': title, 'detail': detail})
                    firstIdx = lastIdx
            else:
#                title = re.sub(r"[:#.,-_]", "", item['title'])
                title = item['title'].replace(':', '').strip()
                chapters.append({'title': title, 'detail': content})
            break   # finished show break retry state
        except:
            traceback.print_exc()
            itime += 1
            print '>>> Retry {0}: {1}'.format(item['title'], url)
            time.sleep(5)
    return chapters


def createTable(c):
    c.execute("DROP TABLE IF EXISTS tblChapters")
    c.execute("DROP TABLE IF EXISTS tblDetail")
    c.execute("CREATE TABLE tblChapters(`id` INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, `title` TEXT)")
    c.execute("CREATE INDEX chapter_idx ON tblChapters(id)")
    c.execute("CREATE TABLE tblDetail(`id` INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, `chapterId` INTEGER, `detail` TEXT)")
    c.execute("CREATE INDEX detail_idx ON tblDetail(chapterId)")  
        
def process():
    filestore = 'ynham_chapters'
    
    try:
        f = open(filestore, 'r')
        items = pickle.load(f)
        f.close()
    except:
        items = None
    if items == None:
        items = getChapterLink()
        f = open(filestore, 'w') 
        pickle.dump(items, f)
        f.close()
    conn = db.connect(DBName)
    c = conn.cursor()
    createTable(c)
    for item in items:
        chapters = getContent(item)
        for chapter in chapters:
            print chapter['title']
            print chapter['detail']
            print '-----------------------------------------------'
            c.execute("INSERT INTO tblChapters(title) VALUES('{0}')".format(chapter['title']).upper())
            chapterId = c.lastrowid
            chapter['detail'] = re.sub(r"\"", "\\\"", chapter['detail'])
            c.execute("INSERT INTO tblDetail(chapterId, detail) VALUES(?, ?)", (chapterId, chapter['detail']))
    c.close()
    conn.commit()
    conn.close()
    

if __name__ == '__main__':
    
    DBName = 'yeunhamchi2'
    process()
    
#    getContent({'title': 'a', 'url': 'http://vozforums.com/showpost.php?p=48731690&postcount=23422'})
    
    print '> Finished at {0}'.format(datetime.datetime.now())
    os._exit(1)
    
    

    